{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ART for Tensorflow v2 - Keras API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook demonstrate applying ART with the new Tensorflow v2 using the Keras API. The code follows and extends the examples on www.tensorflow.org."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.disable_eager_execution()\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from art.classifiers import KerasClassifier\n",
    "from art.attacks import FastGradientMethod, CarliniLInfMethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if tf.__version__[0] != '2':\n",
    "    raise ImportError('This notebook requires Tensorflow v2.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "\n",
    "x_test = x_test[0:100]\n",
    "y_test = y_test[0:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow with Keras API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a model using Keras API. Here we use the Keras Sequential model and add a sequence of layers. Afterwards the model is compiles with optimizer, loss function and metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.InputLayer(input_shape=(28, 28)),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.2),\n",
    "    tf.keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the model on training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples\n",
      "Epoch 1/3\n",
      "60000/60000 [==============================] - 7s 118us/sample - loss: 0.3038 - accuracy: 0.9122\n",
      "Epoch 2/3\n",
      "60000/60000 [==============================] - 7s 112us/sample - loss: 0.1460 - accuracy: 0.9568\n",
      "Epoch 3/3\n",
      "60000/60000 [==============================] - 8s 135us/sample - loss: 0.1107 - accuracy: 0.9664\n"
     ]
    }
   ],
   "source": [
    "model.fit(x_train, y_train, epochs=3);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model accuracy on test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 0s 1ms/sample - loss: 0.0308 - accuracy: 1.0000\n",
      "Accuracy on test data: 100.00%\n"
     ]
    }
   ],
   "source": [
    "loss_test, accuracy_test = model.evaluate(x_test, y_test)\n",
    "print('Accuracy on test data: {:4.2f}%'.format(accuracy_test * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a ART Keras classifier for the Tensorflow Keras model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = KerasClassifier(model=model, clip_values=(0, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fast Gradient Sign Method attack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a ART Fast Gradient Sign Method attack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_fgsm = FastGradientMethod(classifier=classifier, eps=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate adversarial test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_adv = attack_fgsm.generate(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate accuracy on adversarial test data and calculate average perturbation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 0s 302us/sample - loss: 17.4932 - accuracy: 0.0000e+00\n",
      "Accuracy on adversarial test data: 0.00%\n",
      "Average perturbation: 0.18\n"
     ]
    }
   ],
   "source": [
    "loss_test, accuracy_test = model.evaluate(x_test_adv, y_test)\n",
    "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
    "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test * 100))\n",
    "print('Average perturbation: {:4.2f}'.format(perturbation))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualise the first adversarial test sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQD0lEQVR4nO3df2xd9XnH8c+DE3BxoEsIc0MaFkgzrWxTA1hhpQylymgZEQK0DRVNNFOlha2glQ61Q/wD2rQNTYW0k1amUNKGiTIhASVV2EZmdaNTh4WhDPJjbQAFhdTYJB5LsBRInGd/+ITeJb7fY/vc88N+3i8p8vV57rn3ybnOJ+fH199j7i4AcZ1WdwMA6kUIAMERAkBwhAAQHCEABEcIAMHVEgJmdrWZ/cTMXjWzO+voIcXM9prZK2b2kpkNNqCfzWY2YmY7WpYtMrPtZrYn+7qwYf3dY2b7s234kpldU2N/y8zsB2a2y8x2mtmXsuWN2IaJ/irZhlb1OAEz65L0U0lXSXpT0vOSbnL3XZU2kmBmeyX1ufuBunuRJDO7UtK7kh5291/Llv2NpFF3vzcL0oXu/mcN6u8eSe+6+9fq6KmVmS2RtMTdXzSzsyS9IOl6SX+gBmzDRH83qoJtWMeewGpJr7r76+7+vqR/lHRdDX3MGu7+rKTRkxZfJ2lL9niLJn5oatGmv8Zw9yF3fzF7fFjSbklL1ZBtmOivEnWEwFJJ+1q+f1MV/oWnyCU9Y2YvmNmGuptpo9fdh7LHb0nqrbOZNm4zs5ezw4XaDldamdlySRdLGlADt+FJ/UkVbENODE7uCne/RNJvS7o1291tLJ84pmva+O8HJK2QtErSkKT76m1HMrMFkh6XdLu7H2qtNWEbTtJfJduwjhDYL2lZy/cfzZY1hrvvz76OSHpSE4cwTTOcHUueOKYcqbmf/8fdh9193N2PS3pQNW9DM5uviX9gj7j7E9nixmzDyfqrahvWEQLPS1ppZheY2emSPidpaw19TMrMerKTMzKzHkmfkbQjvVYttkpanz1eL+mpGns5xYl/XJkbVOM2NDOT9JCk3e5+f0upEduwXX9VbcPKrw5IUnap4+uSuiRtdve/rLyJNszsQk387y9J8yR9t+7+zOxRSWskLZY0LOluSd+T9Jik8yW9IelGd6/l5Fyb/tZoYjfWJe2VdEvL8XfV/V0h6YeSXpF0PFt8lyaOu2vfhon+blIF27CWEADQHJwYBIIjBIDgCAEgOEIACI4QAIKrNQQaPCRXEv0V1eT+mtybVG1/de8JNPqDEP0V1eT+mtybVGF/dYcAgJoVGixkZldL+oYmRv59y93vTT3/dDvDu9XzwfdH9Z7m64wPvh8/p2ey1T7QdXAsWc9bP8/Jrz/d/spWd3/T3f7HjoxpXvfUeyr6+U5n/cl6y1u/qOn0f/JnO931T3ZEY3rf37PJavOSr5qQTQ7yd2qZHMTMtqYmB+lWjy6ztW1f8511n0y+5y/8w38m63nr5yn79Yuqu7+mb/+y1y+qzv4HvL9trcjhAJODAHNAkRCYDZODAMgx48OBqcoudWyQpG6dWfbbAZimInsCU5ocxN03uXufu/edfKIDQP2KhECjJwcBMDUzPhxw92Nmdpukf9HPJwfZmVpn/Jye5BnO3LOjNxc8u5qzftOV3X/ZZ8fzFP188zT956PMn+/xbc+1rRU6J+DuT0t6ushrAKgXIwaB4AgBIDhCAAiOEACCIwSA4AgBILjShw236jo4lrzWGX0cQN2avv2a3l/dP5+p9+/y9r9mzJ4AEBwhAARHCADBEQJAcIQAEBwhAARHCADBVTpOIE/d11kxt9U9X0XR+RDKen32BIDgCAEgOEIACI4QAIIjBIDgCAEgOEIACK5R4wSKKvs6LGIrexxAUTO97wB7AkBwhAAQHCEABEcIAMERAkBwhAAQHCEABFfpOIHxc3r0zrryrsU3/To/4xjqNde330zvO1AoBMxsr6TDksYlHXP3viKvB6B6ndgT+LS7H+jA6wCoAecEgOCKhoBLesbMXjCzDZ1oCEC1ih4OXOHu+83sFyVtN7P/dvdnW5+QhcMGSTq9Z2HBtwPQaYX2BNx9f/Z1RNKTklZP8pxN7t7n7n3zunuKvB2AEsw4BMysx8zOOvFY0mck7ehUYwCqUeRwoFfSk2Z24nW+6+7/3JGu2qj797XLVvQ6dvRxCHXfVyBPU+cjmHEIuPvrkj7RwV4A1IBLhEBwhAAQHCEABEcIAMERAkBwhAAQnLl7ZW92ti3yy2xt23rT7+9+2plnJutv/OmqZP3MofS2nnckXf/wI+3nju+EuscJzPVxIEUV+Xx2bduosQP7bLIaewJAcIQAEBwhAARHCADBEQJAcIQAEBwhAARX6X0H8jT9OvFPvvnxZH3Fsn2lvv/xLyxL1g+8m5656d3d6end9nz+gWn31Grlw39caP3Re4uNU8jrf9W9X0zWe//2R8l6U+cDKIo9ASA4QgAIjhAAgiMEgOAIASA4QgAIjhAAgmvUfAKznV38q8m6/3hnofXfXn12sv6/Vx5J1i/4SPrm0a//bHGy/trabyfrRV35yg3J+tDBDyfry3sPFnr/z533fLK+6a/S/ZU9TqDIOIUB79chH2U+AQCnIgSA4AgBIDhCAAiOEACCIwSA4AgBIDjGCUzDXP198hNGbr08WX/n0vcLvf4vf2Gw0Pp53nw8Pc5i5ycfSdav+fTvJusHV5877Z5a5f18FL3vQ2njBMxss5mNmNmOlmWLzGy7me3JvqZnqwDQWFM5HPiOpKtPWnanpH53XympP/sewCyUGwLu/qyk0ZMWXydpS/Z4i6TrO9wXgIrM9MRgr7sPZY/fktTboX4AVKzw1QGfOLPY9uyimW0ws0EzGzyq94q+HYAOm2kIDJvZEknKvo60e6K7b3L3Pnfvm68zZvh2AMoy0xDYKml99ni9pKc60w6AquWOEzCzRyWtkbRY0rCkuyV9T9Jjks6X9IakG9395JOHp6h7nEDR67B5mj5OoOy/f1F52++07u5k/Z9efy5Zv/CJW5L1s/Z0Jevdo9WNqem0Xds2auzAvknHCeTefMTdb2pTmr2jfgB8gGHDQHCEABAcIQAERwgAwRECQHCEABBc7iVC/FzTxwE0Xd72G7ojPZ/B+5cdznmH9DiBDy15N1nvHkjf16HpUtu3y8fa1tgTAIIjBIDgCAEgOEIACI4QAIIjBIDgCAEgOMYJTMNcv+9A2fK23+98/t+S9bvP3VXo/Y/uSY8DKPu+AGVL9Te+rf0YCvYEgOAIASA4QgAIjhAAgiMEgOAIASA4QgAILve+A53Us3iZX7Tuy23rXGcvpunXsUcuH0/Wl15wIFn/0Pyjyfppa/dNu6dOKjqOpOjnl3r9Ae/XIR+d9L4D7AkAwRECQHCEABAcIQAERwgAwRECQHCEABBcpfMJdB0cS17L5Pf157Yv/mZ/sv7M8EXJ+pv/vixZP1/1jhOoez6C0uYTMLPNZjZiZjtalt1jZvvN7KXszzXTbRhAM0zlcOA7kq6eZPlGd1+V/Xm6s20BqEpuCLj7s5JGK+gFQA2KnBi8zcxezg4XFnasIwCVmmkIPCBphaRVkoYk3dfuiWa2wcwGzWzwqN6b4dsBKMuMQsDdh9193N2PS3pQ0urEcze5e5+7983XGTPtE0BJZhQCZrak5dsbJO1o91wAzZY7TsDMHpW0RtJiM3tT0t2S1pjZKkkuaa+kW0rsMYymzweQJ+86+d9felWy/pFfGUnWz//zH027p06a7Z9PO7kh4O43TbL4oRJ6AVADhg0DwRECQHCEABAcIQAERwgAwRECQHCVzicwfk6P3llX3rXWsq/jFp3PYK5eZz5hfM0lyfqFv74/Wf/Zv6bnC1ig15P1uuf9n63YEwCCIwSA4AgBIDhCAAiOEACCIwSA4AgBILhZdd+BujW9v7LlXWd/75nlyfr/HF6QrH/0r8udL2C2f35Fxql0+VjbGnsCQHCEABAcIQAERwgAwRECQHCEABAcIQAEV+k4gTxFf18fxRw5x5L1n3770mT9Y13p+wZ8/5IHk/Xfu/kryXp0RedLaIc9ASA4QgAIjhAAgiMEgOAIASA4QgAIjhAAgptT9x1Amnel6//11W8m61ftvjZZf3Vvb7L+2We/mqyfLU/W57q6xsnk7gmY2TIz+4GZ7TKznWb2pWz5IjPbbmZ7sq8Ly28XQKdN5XDgmKQ73P0iSb8h6VYzu0jSnZL63X2lpP7sewCzTG4IuPuQu7+YPT4sabekpZKuk7Qle9oWSdeX1SSA8kzrxKCZLZd0saQBSb3uPpSV3pKUPiAE0EhTDgEzWyDpcUm3u/uh1pq7uzT5WR0z22Bmg2Y2eOxI+8kOAdRjSiFgZvM1EQCPuPsT2eJhM1uS1ZdImvRXyNx9k7v3uXvfvO6eTvQMoIOmcnXAJD0kabe7399S2ippffZ4vaSnOt8egLLZxJ584glmV0j6oaRXJB3PFt+lifMCj0k6X9Ibkm5099HUa51ti/wyWzvjZmf7vPF1y7sOfbx/WbK+/ePfT9Y/e96qaffUaq5/vnXOlzHg/Trko5NOGJE7WMjd/0NSu9kmZv4vGkAjMGwYCI4QAIIjBIDgCAEgOEIACI4QAIJr1H0H5vp14rLlXYfe8/AlyfoKvZ2sX33t7+d0sDNZjf75lnXfgKm8/vi259rW2BMAgiMEgOAIASA4QgAIjhAAgiMEgOAIASC4Rt13IO86ad3Xmcu8jtsJv7XjcLJ+fDg9DiCPv5AeB4Biyh5H0A57AkBwhAAQHCEABEcIAMERAkBwhAAQHCEABNeo+QTKVva870Wv8+atP/Kp8WT9K4teK1TPs+Lrf5Ssf+z29r+zjuLKGmfCngAQHCEABEcIAMERAkBwhAAQHCEABEcIAMHljhMws2WSHpbUK8klbXL3b5jZPZL+UPpgsvq73P3pIs0UvQ7a9N/3L/r61/b9uEOdzEzRcQCzfb6Isn8+y3z9Lh9rW5vKYKFjku5w9xfN7CxJL5jZ9qy20d2/Np1GATRLbgi4+5CkoezxYTPbLWlp2Y0BqMa0zgmY2XJJF0sayBbdZmYvm9lmM1vY4d4AVGDKIWBmCyQ9Lul2dz8k6QFJKySt0sSewn1t1ttgZoNmNnjsSPvjEgD1mFIImNl8TQTAI+7+hCS5+7C7j7v7cUkPSlo92bruvsnd+9y9b153T6f6BtAhuSFgZibpIUm73f3+luVLWp52g6QdnW8PQNmmcnXgU5JulvSKmb2ULbtL0k1mtkoTlw33SrqllA4BlMrcvbI3O9sW+WW2tm29zvu3T0XZ8xHk8cs/ka7/xWiy/trO85L1lX8ykKzPdnXN69+E9x/wfh3yUZusxohBIDhCAAiOEACCIwSA4AgBIDhCAAiOEACCq3ScQM/iZX7Rui/PeP26r9MDdSoyzmXXto0aO7CPcQIATkUIAMERAkBwhAAQHCEABEcIAMERAkBwlY4TMLO3Jb3RsmixpAOVNTB99FdMk/trcm9S5/v7JXc/d7JCpSFwypubDbp7X20N5KC/YprcX5N7k6rtj8MBIDhCAAiu7hDYVPP756G/YprcX5N7kyrsr9ZzAgDqV/eeAICaEQJAcIQAEBwhAARHCADB/R/jzEtlJ9lxSwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Carlini&Wagner Infinity-norm attack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a ART Carlini&Wagner Infinity-norm attack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_cw = CarliniLInfMethod(classifier=classifier, eps=0.3, max_iter=100, learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate adversarial test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_adv = attack_cw.generate(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate accuracy on adversarial test data and calculate average perturbation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 0s 156us/sample - loss: 1.4929 - accuracy: 0.2400\n",
      "Accuracy on adversarial test data: 24.00%\n",
      "Average perturbation: 0.02\n"
     ]
    }
   ],
   "source": [
    "loss_test, accuracy_test = model.evaluate(x_test_adv, y_test)\n",
    "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
    "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test * 100))\n",
    "print('Average perturbation: {:4.2f}'.format(perturbation))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualise the first adversarial test sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAPFklEQVR4nO3dbYxc5XnG8esCL0YYG9syMQbsmLdKkEYYsjUhRciVS+JGQoBUIayUukoVIyUoQUVtEV9AraqiKpDQD6U1xYqhhAgVKCChEselJRHgYBOCbRwwRXaxs9gQ49pQgd/ufthxuoXdZ3b3zJwzu/f/J1k7c+6ZOTdnZy+ec84zZxwRApDXcU03AKBZhACQHCEAJEcIAMkRAkByhACQXCMhYHuZ7ddsv2H7liZ6KLG93fYm2y/b3tAD/ay2vcf25iHLZttea3tb6+esHuvvdtu7WtvwZdtfbrC/+bafsf2q7S22v9Va3hPbsNBfLdvQdc8TsH28pNclXSFpp6QXJS2PiFdrbaTA9nZJ/RHxbtO9SJLtyyW9L+n+iPjN1rK/kbQ3Iu5oBemsiPjzHurvdknvR8S3m+hpKNvzJM2LiJdsT5e0UdLVkv5IPbANC/1dqxq2YRMjgcWS3oiINyPioKQfSLqqgT4mjIh4VtLejy2+StKa1u01GnzTNGKE/npGRAxExEut2wckbZV0hnpkGxb6q0UTIXCGpLeG3N+pGv+DRykk/dD2Rtsrm25mBHMjYqB1+21Jc5tsZgQ32n6ltbvQ2O7KULYXSrpI0nr14Db8WH9SDduQA4PDuywiLpb0e5K+0Rru9qwY3Kfrtfnf90g6R9IiSQOS7my2Hcn2yZIekXRTROwfWuuFbThMf7VswyZCYJek+UPun9la1jMiYlfr5x5Jj2lwF6bX7G7tSx7bp9zTcD//T0TsjogjEXFU0r1qeBva7tPgH9iDEfFoa3HPbMPh+qtrGzYRAi9KOs/2WbZPkHSdpCca6GNYtqe1Ds7I9jRJX5S0ufysRjwhaUXr9gpJjzfYyycc++NquUYNbkPblnSfpK0RcdeQUk9sw5H6q2sb1n52QJJapzq+K+l4Sasj4q9qb2IEts/W4P/9JWmKpO833Z/thyQtkTRH0m5Jt0n6F0kPS1ogaYekayOikYNzI/S3RIPD2JC0XdINQ/a/6+7vMkk/lrRJ0tHW4ls1uN/d+DYs9LdcNWzDRkIAQO/gwCCQHCEAJEcIAMkRAkByhACQXKMh0MNTciXRX1W93F8v9ybV21/TI4Ge/kWI/qrq5f56uTepxv6aDgEADas0Wcj2Mkl3a3Dm3z9GxB2lx5/gqXGipv36/iF9pD5NHff6u43+qunl/nq5N6nz/X2oD3QwPvJwtXGHwHguDjLDs+MSLx3X+gCM3/pYp/2xd9gQqLI7wMVBgEmgSghMhIuDAGhjSrdX0DrVsVKSTtRJ3V4dgDGqMhIY1cVBImJVRPRHRH8vH4gBsqoSAj19cRAAozPu3YGIOGz7RklP6/8uDrKlY50BqEWlYwIR8ZSkpzrUC4AGMGMQSI4QAJIjBIDkCAEgOUIASI4QAJIjBIDkCAEgOUIASI4QAJIjBIDkCAEgOUIASI4QAJLr+uXF0Dn7rr+0WJ/5wPNdfT4mJ0YCQHKEAJAcIQAkRwgAyRECQHKEAJAcIQAkxzyBDnLfCcX6e9d9rlif8287Kq1/ylmfLtaZB4DhMBIAkiMEgOQIASA5QgBIjhAAkiMEgOQIASA5R0RtK5vh2XGJl9a2PgCD1sc67Y+9Hq5WabKQ7e2SDkg6IulwRPRXeT0A9evEjMHfiYh3O/A6ABrAMQEguaohEJJ+aHuj7ZWdaAhAvaruDlwWEbtsf0rSWtu/iIhnhz6gFQ4rJelEnVRxdQA6rdJIICJ2tX7ukfSYpMXDPGZVRPRHRH+fplZZHYAuGHcI2J5me/qx25K+KGlzpxoDUI8quwNzJT1m+9jrfD8i/rUjXQGozbhDICLelHRhB3sB0ABOEQLJEQJAcoQAkBwhACRHCADJEQJAcnzvwBgcd1J52vOOP1lUrJ80UL52w5QPy/VTHnyhWAfGg5EAkBwhACRHCADJEQJAcoQAkBwhACRHCADJMU9gDF77u/OL9XPmv9XV9R/96vxi/d33pxXr72+d1cl2es62P7ynWF90x9eL9bl/+1wn25kwGAkAyRECQHKEAJAcIQAkRwgAyRECQHKEAJCcI8qfYe+kGZ4dl3hpbeurmy/6TLEeP9tS6fnvLJ5RrP/35R8W62edVv7y6Dd/OadYP/v0al8+vfb8J4v1yzddU6wP/OqUYn3h3F+Nuaehrjv9xWL94fNPq/T6TVof67Q/9nq4GiMBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSY54ARu311f2Vnv8bX93QoU6Gt/OR8jyL02fuL9b7vj61WD/y2htj7qlXVJonYHu17T22Nw9ZNtv2WtvbWj8n99UqgElsNLsD35O07GPLbpG0LiLOk7SudR/ABNQ2BCLiWUl7P7b4KklrWrfXSLq6w30BqMl4DwzOjYiB1u23Jc3tUD8Aalb57EAMHlkc8eii7ZW2N9jecEgfVV0dgA4bbwjstj1Pklo/94z0wIhYFRH9EdHfp/LRVwD1G28IPCFpRev2CkmPd6YdAHVr+70Dth+StETSHNs7Jd0m6Q5JD9v+Y0k7JF3bzSbRGz71TF+xPvOB57u6/nbzALZc+mCl179wWfl7CU6bwPMEStqGQEQsH6HErB9gEmDaMJAcIQAkRwgAyRECQHKEAJAcIQAk1/YUIfLYd/2llZ7/4ZWLi/UTn/xpsT5w8xeK9SNHDoy5p6E+8/xXivUz736u0utPVIwEgOQIASA5QgBIjhAAkiMEgOQIASA5QgBIjnkCiRxc9ltdff128wDaOfXl8uXn1t38QKXXP7RtRqXnT1aMBIDkCAEgOUIASI4QAJIjBIDkCAEgOUIASI55Aon8z6nN/rpfv6d8vYEzznq3WL9i65XF+nFL3yrWz1J3vxdhomIkACRHCADJEQJAcoQAkBwhACRHCADJEQJAcswTwKhNGzhY6fkLH49ifcqfHirWd/7H/GJ9gcrzBDC8tiMB26tt77G9eciy223vsv1y69+Xu9smgG4Zze7A9yQtG2b5dyJiUevfU51tC0Bd2oZARDwraW8NvQBoQJUDgzfafqW1uzCrYx0BqNV4Q+AeSedIWiRpQNKdIz3Q9krbG2xvOKTyhSQB1G9cIRARuyPiSEQclXSvpBE/HhYRqyKiPyL6+zR1vH0C6JJxhYDteUPuXiNp80iPBdDb2s4TsP2QpCWS5tjeKek2SUtsL5IUkrZLuqGLPWKUPvj9S7r6+n0/2ljp+f+17Phi/bRDfcX6gr94rtL6Mby2IRARy4dZfF8XegHQAKYNA8kRAkByhACQHCEAJEcIAMkRAkByXE9gEpn2z+uL9X3XX1qsz3yg2nX5jyy5uFg/+7O7ivVf/qh8vYCT9eaYe0J7jASA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOeQKJVJ0H0M7hW8vXo33vwMnF+pl/zfUCmsBIAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5JgnMIEcuO7zxfr0H7xQ6fXfvukLxfr+C8tfI3fu8XuK9ScvvrdY/5ouK9Z7nl2uR9TTxxgxEgCSIwSA5AgBIDlCAEiOEACSIwSA5AgBILnJNU+gzXnafX9QPs/e9vP2iz9brv90U7leUdV5AO18MP9osX7ugvI8gLXnP1msn/8Pf1asL1D5egJTzl5YrB9+c3ux3m2V318NaTsSsD3f9jO2X7W9xfa3Wstn215re1vr56zutwug00azO3BY0s0RcYGkz0v6hu0LJN0iaV1EnCdpXes+gAmmbQhExEBEvNS6fUDSVklnSLpK0prWw9ZIurpbTQLonjEdGLS9UNJFktZLmhsRA63S25LmdrQzALUYdQjYPlnSI5Juioj9Q2sREZKG/XSE7ZW2N9jecEjlD6AAqN+oQsB2nwYD4MGIeLS1eLftea36PEnDHjqOiFUR0R8R/X2a2omeAXTQaM4OWNJ9krZGxF1DSk9IWtG6vULS451vD0C3Odp8xtn2ZZJ+LGmTpGMnkm/V4HGBhyUtkLRD0rURUbzw/AzPjku8tGrPI9p3/aXFetfP007Qz5Mfc3Td/GK93TyAL52+qJPtfMLhpZ8r1qes29jV9U9k62Od9sfeYd+gbScLRcRPJI307u7eXzSAWjBtGEiOEACSIwSA5AgBIDlCAEiOEACSm1TXE5j98/eK9fKn5Tugx+cBbLv/4mL9HL1TrC+78itt1rBljB2NzUczy2/XSfVmrhEjASA5QgBIjhAAkiMEgOQIASA5QgBIjhAAkptQp1b3Ly9f1/3olPLn+We+0slues/vbj5QrD89e3Wl1//Sxu5eL6Cd4w82Ow/juBNPLNbjSHkmShw62Ml2OoaRAJAcIQAkRwgAyRECQHKEAJAcIQAkRwgAyU2oeQIzHnqh6RYa9frfLy7Wn569qqvrf+O75Xka597U3d/PSf++tVhvd72I46ZPLz//QHmeRRw+XKzvu66/WD/ln3rz/ctIAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5NrOE7A9X9L9kuZKCkmrIuJu27dL+pr064vV3xoRT3WrUUhX9v+s0fV3ex5AO+3O43f7+e3mCfTqPIB2RjNZ6LCkmyPiJdvTJW20vbZV+05EfLt77QHotrYhEBEDkgZatw/Y3irpjG43BqAeYzomYHuhpIskrW8tutH2K7ZX257V4d4A1GDUIWD7ZEmPSLopIvZLukfSOZIWaXCkcOcIz1tpe4PtDYf0UQdaBtBJowoB230aDIAHI+JRSYqI3RFxJCKOSrpX0rCfbomIVRHRHxH9fZraqb4BdEjbELBtSfdJ2hoRdw1ZPm/Iw66RtLnz7QHottGcHfhtSddL2mT75dayWyUtt71Ig6cNt0u6oSsdAuiq0Zwd+Imk4S7oz5yAmv3imxcU61f85WnF+n9uOb1YP++b64t1TE7MGASSIwSA5AgBIDlCAEiOEACSIwSA5AgBILkJ9b0D2fm5n5frS8vPP09vdbAbTBaMBIDkCAEgOUIASI4QAJIjBIDkCAEgOUIASM4RUd/K7Hck7RiyaI6kd2trYOzor5pe7q+Xe5M639+nI+LU4Qq1hsAnVm5viIj+xhpog/6q6eX+erk3qd7+2B0AkiMEgOSaDoFVDa+/Hfqrppf76+XepBr7a/SYAIDmNT0SANAwQgBIjhAAkiMEgOQIASC5/wU3CjxsTvsGBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[0, :, :])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv36_TF20",
   "language": "python",
   "name": "venv36_tf20"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
